| linux, shell

Shell Text Stream

Deal text with sort, tail, head, cut, diff, grep, sed, awk.

1 Query with column in text

1.1 Preprocess of text

awk -F '[@-]' '/@gmail.com/{print $1,$3}' access.log | awk -F '[[/=]' '{print $1, $NF}'>ip_user.txt | sort -V | uniq -c > ip_user_sort.txt

1.2 Query IP GeoLocation

awk

### Default delimiter is blankspace.
awk '{print $1}' hosts.txt
awk '{print $1}' hosts.txt | tee ip.txt
awk 'BEGIN {print "IP\n---"} {print $1}' hosts.txt
awk 'BEGIN {print "IP"; print "---"} {print $1} END {print "---"}' hosts.txt
### Define another delimiter with -f 'x'.
awk -F ':' '{print $1}' hosts.txt
awk -F: '{print $1}' hosts.txt
### Define multiple delimiters
awk -F '[:@ /t]' '{print $1}' hosts.txt
### Match strings
awk -F: '/root/' /etc/passwd
awk -F: '/root/{print $1}' /etc/passwd
### Define string delimiter
awk '$1~/^(12)/{print $1}' hosts.txt
## Print matched lines, and set $3
awk '$2=="gmail.com"{$3="gg"; print $1$3}' hosts.txt
## Print all lines, and set $3
awk '$2=="gmail.com"{$3="gg"}{print $1$3}' hosts.txt

Detect Private IP

## Print Lan IP
awk '$1~/^(10\.|192\.168|127\.0)/{print $1}' hosts.txt
## Skip Lan IP
awk '$1!~/^(10\.|192\.168|127)/{print $1}' hosts.txt

Curl

curl http://ip-api.com/json/102.34.12.51
curl https://ipapi.co/102.34.12.51/json
curl https://ip.cn/index.php?ip=102.34.12.51
sleep 0.1
awk '$1!~/^(10\.|192\.168|127)/{system("curl http://ip-api.com/json/"$1); sleep 0.1}' hosts.txt | tee iploc.txt
awk '$1!~/^(10\.|192\.168|127)/{system("curl https://ipapi.co/$1/json"); sleep 0.1}' hosts.txt
awk '$1!~/^(10\.|192\.168|127)/{system("curl https://ip.cn/index.php?ip="$1); sleep 0.1}' hosts.txt

1.3 Query MAC Vendor

curl https://api.macvendors.com/64:76:ba:a6:fe:4a
curl http://macvendors.co/api/64:76:ba:a6:fe:4a
curl http://macvendors.co/api/64:76:ba:a6:fe:4a/pipe
arp -a | awk -F '[() ]' '$0!~/^(\?)/{print $6}'
arp -a | awk -F '[() ]' '$0!~/^(\?)/{system("curl https://api.macvendors.com/"$6); sleep 0.1}'
arp -a | awk -F '[() ]' '$0!~/^(\?)/{system("curl http://macvendors.co/api/vendorname/"$6); sleep 0.1}'
arp -a | awk -F '[() ]' '$0!~/^(\?)/{system("curl http://macvendors.co/api/"$6); sleep 0.1}'